#coding:utf-8
"""
*   TG_Coder : Dzlua
*   Email    : 505544956@qq.com
*   Time     : 2017/08/16
"""

from bs4 import BeautifulSoup
import demjson

from spider import Spider
from database import DataBase

import codecs

#-----------------------
class LibDir(Spider, DataBase):
    def __init__(self, host, user, pwd, db, threads_work = 4, tasks):
        self.__tb = 'libdir';
        DataBase.__init__(self, host, user, pwd, db)
        sql = 'CREATE TABLE IF NOT EXISTS ' + self.__tb + \
            ' (id INTEGER AUTO_INCREMENT,name TEXT,href TEXT,type TEXT,active TINYINT,date TIMESTAMP, PRIMARY KEY(id) ) \
            DEFAULT CHARSET=utf8;'
        DataBase.execute(self, sql)

        Spider.__init__(self, threads_work, tasks)

        self.__url = 'https://www.oschina.net'

        self.put(self.__on_page, self.__url + '/project')
    def __on_save(self, conn, data, args):
        sql = "INSERT INTO %s (name,href,type,active) VALUES('%s','%s','%s','%d');" % (self.__tb, data['name'], data['href'], data['type'], data['active'])
        self.execute(sql)
        print(data)
    def __on_page(self, ret, url, args):
        if not ret:
            return

        soup = BeautifulSoup(ret, "lxml")
        div = soup.find(id="v-sort")
        if not div:
            print('[LibDir] %s: no #v-sort' % (url))
            return
        div = div.find('project-sort')
        if not div:
            print('[LibDir] %s: no #project-sort' % (url))
            return
        
        text = div.prettify()
        str_b = text.find(':param="')
        str_e = text.find('">')
        text = text[str_b + len(':param="') : str_e]

        self.__parser_json(text)

    def __parser_json(self, text):
        js = demjson.decode(text)
        if not js or not js['menu'] or not js['menu']['items']:
            return
        for item in js['menu']['items']:
            for child in item['childs']:
                data = {}
                data['active'] = item['active']
                data['type'] = self.strip(item['name'])
                data['name'] = self.strip(child['name'])
                data['href'] = self.strip(self.__url + child['href'])
                self.save(self.__on_save, data)
    def write(self, text):
        f = codecs.open(self.__tb + '.html', 'w', 'utf-8')
        f.write(text)
        f.close()
#-----------------------
#libdir = LibDir('localhost', 'dzlua', 'dzlua', 'spider', 4, 10)
#libdir.wait()
#libdir.close()